import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import urllib.request
import os
import json
import demjson
import re
from bs4 import BeautifulSoup
import requests
import traceback
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

# 普通大众版本
class common:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        regex = r'column/([\d]*)\?type'
        catid = re.findall(regex, url)
        if (catid != '' and catid != None):
            self.catId = int(catid[0])
            orgul = url.split("type=4")[0]
            self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        # time.sleep(3)
        # browser.save_screenshot('baid.png')
        # 获取siteId
        siteId = browser.execute_script("return global_siteId")
        # print("siteId:", siteId)
        # 获取catId

        # 获取json
        jsonLink = ''
        jsonpath = 'json/' + str(siteId) + '.json'
        if (os.path.exists(jsonpath)):
            with open(jsonpath, 'r') as load_f:
                jsonLink = json.load(load_f)
        else:
            a = urllib.request.urlopen(self.host + "/" + str(siteId) + ".js")  # 打开指定网址
            html = a.read()  # 读取网页源码
            html = html.decode("utf-8")  # 解码为unicode码
            # print(html)
            # html = html.strip('''"var catalogs = ''')
            jsonLink = html[15:]
            # jsonLink = demjson.decode(html)
            with open(jsonpath, "w", encoding='utf-8') as f:
                json.dump(jsonLink, f)
        # key是数字时使用此方法
        jsonLink = demjson.decode(jsonLink)
        urllinks = []

        # 组配分类
        # 获取当前网站的树链接集合
        try:
            linkSarr = jsonLink[self.catId]
            # print(linkSarr)
            urllinks = self.RefineLinks(linkSarr, '首页')
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')

        #主题分类
        if self.orgul !='':
            # html = browser.page_source
            # soup = BeautifulSoup(html, 'html.parser')
            # content = soup.find_all(id='ftd1')
            cont = browser.find_elements_by_id('ftd1')
            # find_element_by_id('ftd1')
            # print(cont.text)
            try:
                cont = browser.find_element_by_id('ftd1')
                # if len(cont) > 0 and cont != None:

                if cont.text=='主题分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                    nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree").getNodes()')
                    # print(nodes)
                    for info in nodes:

                        link = {}
                        link['url'] = self.orgul + str(info['id'])
                        title =  '首页-主题分类'+'-' + info['name']
                        link['surl'] = self.url
                        link['title'] = title

                        urllinks.append(link)
            except:
                print()

        browser.quit()
        return urllinks

    #组配分类
    def RefineLinks(self, jsonLink, title):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['parentId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-组配分类'+'-'+title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            uri = info['uri']
            if (uri != None and uri.startswith('http://')):
                continue
            else:
                link['url'] = self.host + info['uri']

            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title


# chro = common('萧县城市管理局','http://zwgk.lingbi.gov.cn/public/column/3714722?type=4&action=list')
# urllinks  = chro.do()
# print(urllinks)


# 宿州埇桥
class szyq:

    def __init__(self, name, url, catid):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        self.catId = catid
        self.orgul = 'szyq.gov.cn/public/column/11?type=4&cId='

    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        # time.sleep(3)
        # browser.save_screenshot('baid.png')
        # 获取siteId
        siteId = browser.execute_script("return global_siteId")
        # print("siteId:", siteId)

        # 获取json
        jsonLink = ''
        jsonpath = 'json/' + str(siteId) + '.json'
        if (os.path.exists(jsonpath)):
            with open(jsonpath, 'r') as load_f:
                jsonLink = json.load(load_f)
        else:
            a = urllib.request.urlopen(self.host + "/" + str(siteId) + ".js")  # 打开指定网址
            html = a.read()  # 读取网页源码
            html = html.decode("utf-8")  # 解码为unicode码
            # print(html)
            # html = html.strip('''"var catalogs = ''')
            jsonLink = html[15:]
            # jsonLink = demjson.decode(html)
            with open(jsonpath, "w", encoding='utf-8') as f:
                json.dump(jsonLink, f)
        # key是数字时使用此方法
        jsonLink = demjson.decode(jsonLink)
        urllinks = []
        # 获取当前网站的树链接集合
        try:
            linkSarr = jsonLink[self.catId]
            # print(linkSarr)
            urllinks = self.RefineLinks(linkSarr, '首页')
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')

        # 主题分类
        if self.orgul != '':
            # html = browser.page_source
            # soup = BeautifulSoup(html, 'html.parser')
            # content = soup.find_all(id='ftd1')
            cont = browser.find_elements_by_id('ftd1')
            # find_element_by_id('ftd1')
            # print(cont.text)
            try:
                cont = browser.find_element_by_id('ftd1')
                # if len(cont) > 0 and cont != None:

                if cont.text == '主题分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                    nodes = browser.execute_script(
                        'return jQuery.fn.zTree.getZTreeObj("organ_class_3032980_tree").getNodes()')
                    # print(nodes)
                    for info in nodes:
                        link = {}
                        link['url'] = self.orgul + str(info['id'])
                        title = '首页-主题分类' + '-' + info['name']
                        link['surl'] = self.url
                        link['title'] = title

                        urllinks.append(link)
            except:
                print()
        browser.quit()
        return urllinks

    def RefineLinks(self, jsonLink, title):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['parentId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-组配分类'+'-'+title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            uri = info['uri']
            if (uri != None and uri.startswith('http://')):
                continue
            else:
                link['url'] = self.host + info['uri']
            link['url'] = self.host + info['uri']
            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            title = titles + "-" + title
        return title


class sixian:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        regex = r'column/([\d]*)\?type'
        catid = re.findall(regex, url)
        if (catid != '' and catid != None):
            self.catId = int(catid[0])
            orgul = url.split("type=4")[0]
            self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'd:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        # time.sleep(3)
        # browser.save_screenshot('baid.png')
        # 获取siteId
        siteId = browser.execute_script("return global_siteId")
        # print("siteId:", siteId)
        # 获取catId

        # 获取json
        jsonLink = ''
        jsonpath = 'json/' + str(siteId) + '.json'
        if (os.path.exists(jsonpath)):
            with open(jsonpath, 'r') as load_f:
                jsonLink = json.load(load_f)
        else:
            a = urllib.request.urlopen(self.host + "/" + str(siteId) + ".js")  # 打开指定网址
            html = a.read()  # 读取网页源码
            html = html.decode("utf-8")  # 解码为unicode码
            # print(html)
            # html = html.strip('''"var catalogs = ''')
            jsonLink = html[15:]
            # jsonLink = demjson.decode(html)
            with open(jsonpath, "w", encoding='utf-8') as f:
                json.dump(jsonLink, f)
        # key是数字时使用此方法
        jsonLink = demjson.decode(jsonLink)
        urllinks = []

        # 组配分类
        # 获取当前网站的树链接集合
        try:
            linkSarr = jsonLink[self.catId]
            # print(linkSarr)
            urllinks = self.RefineLinks(linkSarr, '首页')
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')

        #主题分类
        if self.orgul !='':
            try:
                cont = browser.find_element_by_id('ftd1')
                # if cont.text=='主题分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_2670456_tree").getNodes()')
                    # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-主题分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有主题分类')
            try:
                cont = browser.find_element_by_id('ftd2')
                # if cont.text=='体裁分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_2670481_tree").getNodes()')
                # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-体裁分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有体裁分类')

        browser.quit()
        return urllinks

    #组配分类
    def RefineLinks(self, jsonLink, title):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['parentId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-组配分类'+'-'+title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            uri = info['uri']
            if 'link' in info:
                link['url'] = info['link']
            else:
                if (uri != None and uri.startswith('http://')):
                    continue
                else:
                    link['url'] = self.host + info['uri']

            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title

# chro = sixian('会龙镇','http://www.funan.gov.cn/public/column/4665941?type=4&action=list')
# urllinks  = chro.do()
# print(urllinks)

class quanjiao:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        regex = r'column/([\d]*)\?type'
        catid = re.findall(regex, url)
        if (catid != '' and catid != None):
            self.catId = int(catid[0])
            orgul = url.split("type=4")[0]
            self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        # time.sleep(3)
        # browser.save_screenshot('baid.png')
        # 获取siteId
        siteId = browser.execute_script("return global_siteId")
        # print("siteId:", siteId)
        # 获取catId

        # 获取json
        jsonLink = ''
        jsonpath = 'json/' + str(siteId) + '.json'
        if (os.path.exists(jsonpath)):
            with open(jsonpath, 'r') as load_f:
                jsonLink = json.load(load_f)
        else:
            a = urllib.request.urlopen(self.host + "/" + str(siteId) + ".js")  # 打开指定网址
            html = a.read()  # 读取网页源码
            html = html.decode("utf-8")  # 解码为unicode码
            # print(html)
            # html = html.strip('''"var catalogs = ''')
            jsonLink = html[15:]
            # jsonLink = demjson.decode(html)
            with open(jsonpath, "w", encoding='utf-8') as f:
                json.dump(jsonLink, f)
        # key是数字时使用此方法
        jsonLink = demjson.decode(jsonLink)
        urllinks = []

        # 组配分类
        # 获取当前网站的树链接集合
        try:
            linkSarr = jsonLink[self.catId]
            # print(linkSarr)
            urllinks = self.RefineLinks(linkSarr, '首页')
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')

        #主题分类
        if self.orgul !='':
            try:
                cont = browser.find_element_by_id('ftd1')
                # if cont.text=='主题分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_11414921_tree").getNodes()')
                    # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-主题分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有主题分类')
            try:
                cont = browser.find_element_by_id('ftd2')
                # if cont.text=='体裁分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_11414963_tree").getNodes()')
                # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-体裁分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有体裁分类')

            try:
                cont = browser.find_element_by_id('ftd3')
                # if cont.text=='体裁分类':
                # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_11414944_tree").getNodes()')
                # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title = '首页-按服务对象分类' + '-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有按服务对象分类')

        browser.quit()
        return urllinks

    #组配分类
    def RefineLinks(self, jsonLink, title):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['parentId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-组配分类'+'-'+title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            uri = info['uri']
            if (uri != None and uri.startswith('http://')):
                continue
            else:
                link['url'] = self.host + info['uri']

            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title


class tunxi:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        time.sleep(3)
        branchId = browser.execute_script("return jQuery('#treeDemo').attr('data-branchId')")
        print("branchId:",branchId)
        urllinkall = []
        if branchId!=None:
            while True:
                url = self.host+"/OpennessTree/showBranchTree?branchId=" + branchId
            # print(url)
                rep = requests.post(url=url,headers={'Content-Type': 'application/json'})
                if rep.status_code==200:
                    break
                else:
                    time.sleep(3)
            # print(rep.text)
            arra = eval(rep.text)
            urllinks = self.RefineLinks(arra, branchId)
            urllinkall.extend(urllinks)

        branchId1 = browser.execute_script("return jQuery('#treeDemo_emphasis').attr('data-branchId')")
        if branchId1!=None:
            while True:
                url = self.host + "/OpennessTree/emphasisColumnTree?branchId=" + branchId1
                rep = requests.post(url=url, headers={'Content-Type': 'application/json'})
                if rep.status_code == 200:
                    break
                else:
                    time.sleep(3)
            # print(rep.text)
            arra = eval(rep.text)
            urllinks = self.zdLinks(arra, branchId1)
            urllinkall.extend(urllinks)

        print('抓取完成:', self.name, self.url)
        browser.quit()
        return urllinkall

    #政府信息公开目录
    def RefineLinks(self, jsonLink, branchId):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['pId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-政府信息公开目录'+'-'+title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            link['url'] = self.host + '/BranchOpennessContent/showList/' + branchId + '/' + str(info['code']) + '/page_1.html'

            urllinks.append(link)
        return urllinks

        # 重点信息公开
    def zdLinks(self, jsonLink, branchId):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['pId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-重点信息公开' + '-' + title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            link['url'] = self.host +  '/BranchOpennessContent/showEmphasisColumnList/' + branchId + '/' + str(info['id']) + '/page_1.html'

            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['pId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title


class mingguang:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        regex = r'column/([\d]*)\?type'
        catid = re.findall(regex, url)
        if (catid != '' and catid != None):
            self.catId = int(catid[0])
            orgul = url.split("type=4")[0]
            self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        # time.sleep(3)
        # browser.save_screenshot('baid.png')
        # 获取siteId
        siteId = browser.execute_script("return global_siteId")
        # print("siteId:", siteId)
        # 获取catId

        # 获取json
        jsonLink = ''
        jsonpath = 'json/' + str(siteId) + '.json'
        if (os.path.exists(jsonpath)):
            with open(jsonpath, 'r') as load_f:
                jsonLink = json.load(load_f)
        else:
            a = urllib.request.urlopen(self.host + "/" + str(siteId) + ".js")  # 打开指定网址
            html = a.read()  # 读取网页源码
            html = html.decode("utf-8")  # 解码为unicode码
            # print(html)
            # html = html.strip('''"var catalogs = ''')
            jsonLink = html[15:]
            # jsonLink = demjson.decode(html)
            with open(jsonpath, "w", encoding='utf-8') as f:
                json.dump(jsonLink, f)
        # key是数字时使用此方法
        jsonLink = demjson.decode(jsonLink)
        urllinks = []

        # 组配分类
        # 获取当前网站的树链接集合
        try:
            linkSarr = jsonLink[self.catId]
            # print(linkSarr)
            urllinks = self.RefineLinks(linkSarr, '首页')
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')

        #主题分类
        if self.orgul !='':
            try:
                cont = browser.find_element_by_id('ftd1')
                # if cont.text=='主题分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_1_tree").getNodes()')
                    # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-主题分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有主题分类')
            try:
                cont = browser.find_element_by_id('ftd2')
                # if cont.text=='体裁分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_4_tree").getNodes()')
                # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-体裁分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有体裁分类')

        browser.quit()
        return urllinks

    #组配分类
    def RefineLinks(self, jsonLink, title):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['parentId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-组配分类'+'-'+title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            uri = info['uri']
            if (uri != None and uri.startswith('http://')):
                continue
            else:
                link['url'] = self.host + info['uri']

            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title


class yingzhou:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        regex = r'column/([\d]*)\?type'
        catid = re.findall(regex, url)
        if (catid != '' and catid != None):
            self.catId = int(catid[0])
            orgul = url.split("type=4")[0]
            self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        # time.sleep(3)
        # browser.save_screenshot('baid.png')
        # 获取siteId
        siteId = 2654440#2661049
        # print("siteId:", siteId)
        # 获取catId

        # 获取json
        jsonLink = ''
        jsonpath = 'json/' + str(siteId) + '.json'
        if (os.path.exists(jsonpath)):
            with open(jsonpath, 'r') as load_f:
                jsonLink = json.load(load_f)
        else:
            a = urllib.request.urlopen(self.host + "/" + str(siteId) + ".js")  # 打开指定网址
            html = a.read()  # 读取网页源码
            html = html.decode("utf-8")  # 解码为unicode码
            # print(html)
            # html = html.strip('''"var catalogs = ''')
            jsonLink = html[15:]
            # jsonLink = demjson.decode(html)
            with open(jsonpath, "w", encoding='utf-8') as f:
                json.dump(jsonLink, f)
        # key是数字时使用此方法
        jsonLink = demjson.decode(jsonLink)
        urllinks = []

        # 组配分类
        # 获取当前网站的树链接集合
        try:
            linkSarr = jsonLink[self.catId]
            # print(linkSarr)
            urllinks = self.RefineLinks(linkSarr, '首页')
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')

        #主题分类
        if self.orgul !='':
            try:
                cont = browser.find_element_by_id('ftd2')
                # if cont.text=='主题分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_42_tree").getNodes()')
                    # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-主题分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有主题分类')

        browser.quit()
        return urllinks

    #组配分类
    def RefineLinks(self, jsonLink, title):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['parentId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-组配分类'+'-'+title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            uri = info['uri']
            if (uri != None and uri.startswith('http://')):
                continue
            else:
                link['url'] = self.host + info['uri']

            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title

# chro = yingzhou('会龙镇','http://www.yingzhou.gov.cn/public/column/2661007?type=4&action=list')
# urllinks  = chro.do()
# print(urllinks)


class type2:
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        self.column_page_method = "showList"
        # regex = r'showList/([\d]*)\?0'
        # catid = re.findall(regex, url)
        # if (catid != '' and catid != None):
        #     self.catId = int(catid[0])
        #     orgul = url.split("type=4")[0]
        #     self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        time.sleep(5)

        # WebDriverWait(browser, 60).until( EC.presence_of_all_elements_located((By.ID, "ztree")))
        branchId =  browser.execute_script('return $("#tree").data("branchid")')
        print("branchId:"+str(branchId))
        urllinks = []
        try:
            nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("tree").getNodes()')
            urllinks = self.getLinks(nodes,branchId,'首页-组配分类')
        except:
            print("组配分类异常")
            traceback.print_exc()

        browser.close()
        browser.quit()
        return urllinks

    def getLinks(self,nodes,branchId,otitle):
        urllinks = []
        for info in nodes:
            link = {}
            link['url'] = self.host+"/OpennessContent/showList/" + str(branchId) + "/" + str(info['code']) + "/page_1.html#code"
            title = otitle + '-' + info['name']
            link['surl'] = self.url
            link['title'] = title
            urllinks.append(link)
            if 'children' in info:
                urllinks.extend(self.getLinks(info['children'],branchId,title))
        return urllinks

class type3 :
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        self.column_page_method = "showList"
        # regex = r'showList/([\d]*)\?0'
        # catid = re.findall(regex, url)
        # if (catid != '' and catid != None):
        #     self.catId = int(catid[0])
        #     orgul = url.split("type=4")[0]
        #     self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        time.sleep(5)

        # WebDriverWait(browser, 60).until( EC.presence_of_all_elements_located((By.ID, "ztree")))
        branchId =  browser.execute_script('return $("#treeDemoTopic").attr("data-branchId")')
        print("branchId:"+str(branchId))
        urllinks = []
        try:
            nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("treeDemo").getNodes()')
            urllinks = self.getLinks(nodes,branchId,'首页-组配分类')
        except:
            print("组配分类异常")
            traceback.print_exc()
        try:
            nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("treeDemoTopic").getNodes()')
            urllinks.extend(self.getLinks(nodes, branchId, '首页-主题分类'))
        except:
            print("主题分类异常")
            traceback.print_exc()

        browser.close()
        browser.quit()
        return urllinks

    def getLinks(self,nodes,branchId,otitle):
        urllinks = []
        for info in nodes:
            link = {}
            link['url'] = self.host+"/OpennessContent/showList/" + str(branchId) + "/" + str(info['code']) + "/page_1.html#code"
            title = otitle + '-' + info['name']
            link['surl'] = self.url
            link['title'] = title
            urllinks.append(link)
            if 'children' in info:
                urllinks.extend(self.getLinks(info['children'],branchId,title))
        return urllinks


class type1:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        regex = r'column/([\d]*)\?type'
        catid = re.findall(regex, url)
        if (catid != '' and catid != None):
            self.catId = int(catid[0])
            orgul = url.split("type=4")[0]
            self.orgul = orgul+'type=4&cId='


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        time.sleep(5)

        nodes = browser.execute_script("return jQuery.fn.zTree.getZTreeObj('organ_catalog_tree').getNodes()")
        urllinks = []

        # 组配分类
        # 获取当前网站的树链接集合
        try:
            # print(linkSarr)
            urllinks = self.RefineLinks(nodes, '首页-组配分类')
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')

        #主题分类
        if self.orgul !='':
            try:
                cont = browser.find_element_by_id('ftd1')
                # if cont.text=='主题分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_2670456_tree").getNodes()')
                    # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-主题分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有主题分类')
            try:
                cont = browser.find_element_by_id('ftd2')
                # if cont.text=='体裁分类':
                    # browser.execute_script('var ztree=jQuery.fn.zTree.getZTreeObj("organ_class_3430504_tree")')
                nodes = browser.execute_script('return jQuery.fn.zTree.getZTreeObj("organ_class_2670481_tree").getNodes()')
                # print(nodes)
                for info in nodes:
                    link = {}
                    link['url'] = self.orgul + str(info['id'])
                    title =  '首页-体裁分类'+'-' + info['name']
                    link['surl'] = self.url
                    link['title'] = title
                    urllinks.append(link)
            except:
                print('没有体裁分类')

        browser.quit()
        return urllinks

    #组配分类
    def RefineLinks(self, nodes, otitle):
        urllinks = []
        for info in nodes:
            link = {}
            if 'link' in info:
                link['url'] = info['link']
            else:
                link['url'] = link['url'] = self.host + info['uri']
            title = otitle + '-' + info['name']
            link['surl'] = self.url
            link['title'] = title
            urllinks.append(link)
            if 'children' in info:
                urllinks.extend(self.RefineLinks(info['children'], title))
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title

class gaoxin:
    # 这个是一个用来控制chrome以无界面模式打开的浏览器
    # 创建一个参数对象，用来控制chrome以无界面的方式打开
    def __init__(self, name, url):
        self.name = name
        self.url = url
        # http://zwgk.lingbi.gov.cn/public/column/3714653?type=4&catId=3684985&action=list
        host = url.split("http://")[1]
        host = host.split("/")[0]
        host = "http://" + host
        self.host = host
        regex = r'column/([\d]*)\?type'
        catid = re.findall(regex, url)
        if (catid != '' and catid != None):
            self.catId = int(catid[0])


    def do(self):
        chrome_options = Options()
        # 后面的两个是固定写法 必须这么写
        chrome_options.add_argument('--headless')
        chrome_options.add_argument('--disable-gpu')
        # 驱动路径 谷歌的驱动存放路径
        path = r'f:\chromedriver.exe'
        # 创建浏览器对象
        browser = webdriver.Chrome(executable_path=path, chrome_options=chrome_options)
        browser.get(self.url)
        time.sleep(3)

        nodes = browser.execute_script("return catalogs")
        print(nodes[str(self.catId)])
        # jsonlinks = demjson.decode(nodes[str(self.catId)])
        urllinks = []

        # 组配分类
        # 获取当前网站的树链接集合
        try:
            # print(linkSarr)
            urllinks = self.RefineLinks(nodes[str(self.catId)])
            # for link in urllinks:
            #     print(link['title'],link['url'])
            # 退出浏览器
            print('抓取完成:', self.name, self.url)
        except KeyError:
            print(self.name + ':catId:' + str(self.catId) + '没有找到')
            traceback.print_exc()

        browser.quit()
        return urllinks

    #组配分类
    def RefineLinks(self, jsonLink):
        urllinks = []
        for info in jsonLink:
            # if(info['isParent']==False):
            parentId = info['parentId']
            title = self.getTitle(jsonLink, parentId, info['name'])
            title = '首页-组配分类' + '-' + title
            link = {}
            link['surl'] = self.url
            link['title'] = title  # self.name+'-'+title
            uri = info['uri']
            if (uri != None and uri.startswith('http://')):
                continue
            else:
                link['url'] = self.host + info['uri']

            urllinks.append(link)
        return urllinks

    def getTitle(self, jsonLink, id, name):
        title = name
        item = []
        flag = False
        for info in jsonLink:
            if (info['id'] == id):
                item = info
                flag = True
                break
        titles = ''
        if flag:
            titles = self.getTitle(jsonLink, item['parentId'], item['name'])

        if (titles != ''):
            # title = title + "-" + titles
            title = titles + "-" + title
        return title


ces = type1('民政局','http://www.mingguang.gov.cn/public/column/47068841?type=4&action=list')
urllinks = ces.do()
print(urllinks)

